import datetime
from html.parser import HTMLParser
import json
import efinance as ef
from stock.base.base_east_money import BaseEastMoneyWorker
from enum import Enum
from utils.log_util import saveLog
from utils.time_util import MMDDHHMM, YYYYMMDDHHMMSS, stamp_time, YYYYMMDDHHMM
import difflib
import re


class StockPlatForm(Enum):
    XQ = 1  # 雪球网
    TG = 2  # 淘股吧


class BaseSpiderWorker(BaseEastMoneyWorker):
    # 淘股吧
    taoguBase = "https://www.taoguba.com.cn/"
    taoguUrl = taoguBase + "bbs/"
    taoguEndUrl = "/0"

    # 淘股吧人气热榜
    hotUrl = "https://www.taoguba.com.cn/new/nrnt/getNoticeStock?type=D"

    # 获取几天内的数据
    dayLimit = 3
    # 分析所需数据量
    maxCount = 1000
    # 请求次数超过该值时取消
    cancelLimit = 10
    # 当前请求次数
    requestCount = 0

    # 雪球网
    baseXqUrl = "https://xueqiu.com"
    xqUrl = baseXqUrl + "/statuses/hot/listV2.json?since_id=-1&max_id="
    xqEndUrl = "&size=15"

    # 最大请求数量
    maxArticle = 100
    currentCount = 0
    startTime = 0

    def __init__(self):
        super().__init__()

    def getWebInfo(self, platform):
        """
        1、爬取文章url
        """

        if platform == StockPlatForm.XQ:
            self.xueqiu("雪球网")
            self.getArticle(platform)
        elif platform == StockPlatForm.TG:
            self.taoguba("淘股吧")
            self.getArticle(platform)
            return self.codeFind("淘股吧")

    def getHotStock(self):
        """
        获取淘股吧热门股票
        """
        hotList = []

        try:
            hotResult = self.request.get(self.hotUrl)
            stringValue = hotResult.decode('utf-8')
            obj = json.loads(stringValue)
            for item in obj["dto"]:
                stockInfo = item["stockName"]+"  " + \
                    item["fullCode"] + ("  "+item["linkingBoard"]
                                        if item["linkingBoard"] != "" else "")
                hotList.append(stockInfo)
        except Exception as e:
            print("获取热门股票代码失败：" + str(e))
            saveLog("获取热门股票代码失败")
        return hotList

    def updateStockCode(self):
        """
        更新股票代码
        """
        try:
            codeMap = {}
            resultForMap = {}

            codeDb = self.sqlite.query_many("SELECT * FROM code_stock")
            if len(codeDb) != 0:
                codeMap = {item[2]: item[1] for item in codeDb}

            result = ef.stock.get_realtime_quotes()

            for item in range(0, len(result)):
                itemInfo = result.iloc[item]

                if itemInfo.股票代码 not in resultForMap.keys():
                    resultForMap[itemInfo.股票代码] = itemInfo.股票名称

            # 存在数据库中没有的代码时插入数据库
            resultDict = {
                key: resultForMap[key]
                for key in resultForMap if key not in codeMap
            }

            # 删除无效股票代码
            invalidCode = [
                key[2] for key in codeDb if key[2] not in resultForMap.keys()
            ]
            for item in invalidCode:
                self.sqlite.delete_record(
                    "DELETE FROM code_stock WHERE code = " + item)

            if len(resultDict) != 0:
                saveLog("开始更新股票代码...")
                insertValueList = []
                maxId = self.getMaxId(tableName="code_stock")
                for key, value in resultDict.items():
                    maxId = maxId + 1
                    insertValueList.append(tuple([maxId, value, key]))

                self.sqlite.operate_many(
                    'insert into code_stock values (?,?,?)', insertValueList)

                # 删除重复值
                self.sqlite.delete_record(
                    "DELETE FROM code_stock WHERE rowid NOT IN (SELECT MIN(rowid) FROM code_stock GROUP BY code);"
                )
                saveLog("股票代码更新完成")
            return [key for key in resultForMap.keys()]
        except Exception as e:
            print("获取股票代码失败：" + str(e))
            saveLog("获取股票代码失败：" + str(e))
            return []

    def taoguba(self, platform, page=1):
        """
        淘股吧
        """

        maxId = self.getMaxId(tableName="article_stock")
        currentArticle = self.sqlite.query_many(
            "SELECT url FROM article_stock")

        url = self.taoguUrl + str(page) + self.taoguEndUrl
        content = self.request.get(url, isBytes=False)
        articleList = self.request.getXpath(
            '//div[@class="Nbbs-middle-list"]//div[@class="Nbbs-tiezi-lists"]',
            content)
        for index in range(0, len(articleList)):
            try:
                item = articleList[index]
                childItem = self.request.getXpath(
                    '//div[@class="left middle-list-tittle fs15 c333 cursor overhide"]',
                    item)
                title = self.request.getXpath(
                    '//a[@class="overhide mw300"]//font/text()', childItem[0])
                if len(title) == 0:
                    title = self.request.getXpath(
                        '//a[@class="overhide mw300"]/text()', childItem[0])
                url = self.taoguBase + \
                    self.request.getXpath('//a/@href', childItem[0])[0]
                time = self.request.getXpath(
                    '//div[@class="left middle-list-post cursor"]/text()',
                    item)[0]

                if len([item for item in currentArticle
                        if item[0] == url]) == 0:
                    # 将文章发布时间转换为时间戳
                    currentTime = datetime.datetime.now()
                    timeNow = str(currentTime.year) + "-" + time
                    articleTime = stamp_time(timeNow, YYYYMMDDHHMM)

                    if articleTime < datetime.datetime.now().timestamp(
                    ) and articleTime > self.getAgoTime():
                        title = (title[0] if len(title) != 0 else "").replace(
                            " ", "").replace("\n", "")

                        self.sqlite.operate_one(
                            'insert into article_stock values(?,?,?,?,?,?,?,?)',
                            (maxId, title, url, "", "", "", platform,
                             articleTime))
                        maxId = maxId + 1

                    if index == len(articleList) - 1:
                        count = len(
                            self.sqlite.query_many(
                                "SELECT * FROM article_stock"))
                        if count < self.maxCount and self.requestCount < self.cancelLimit:
                            self.requestCount = self.requestCount + 1
                            self.taoguba(platform, page=page + 1)
            except Exception as e:
                print("淘股吧数据获取失败：" + str(e))
                saveLog("淘股吧数据获取失败")

    def getStartTime(self):
        """
        获取文章爬取开始时间
        """
        currentTime = datetime.datetime.now()
        threeDaysAgo = currentTime - datetime.timedelta(days=3)
        return int(threeDaysAgo.timestamp())

    def getAgoTime(self):
        """
        获取指定天数前的数据
        """
        currentTime = datetime.datetime.now()
        daysAgo = currentTime - datetime.timedelta(days=self.dayLimit)
        return int(daysAgo.timestamp())

    def xueqiu(self, platform, page=0):
        """
        雪球网
        """

        obj = None
        try:
            url = self.xqUrl + str(page) + self.xqEndUrl
            response = self.request.get(url)
            stringData = response.decode('utf-8')
            obj = json.loads(stringData)
        except Exception as e:
            print("雪球网数据获取失败" + str(e))
            saveLog("雪球网数据获取失败")

        if obj is not None and "items" in obj:
            listItem = obj["items"]
            for index in range(0, len(listItem)):
                item = listItem[index]["original_status"]

                currentStamp = stamp_time(
                    str(datetime.datetime.now().year) + "-" +
                    self.timeChange(item["timeBefore"]), YYYYMMDDHHMM)

                if self.currentCount >= self.maxArticle:
                    return

                title = item["title"]
                url = self.baseXqUrl + item["target"]
                maxId = self.sqlite.query_one(
                    "SELECT MAX(id) FROM article_stock")
                id = 0
                if len(maxId) != 0 and maxId[0] is not None:
                    id = maxId[0] + 1

                self.sqlite.operate_one(
                    'insert into article_stock values(?,?,?,?,?,?,?,?)',
                    (id, title, url, "", "", "", platform, currentStamp))
                self.currentCount = self.currentCount + 1

                if index == len(listItem) - 1:
                    page = obj["next_max_id"]
                    self.xueqiu(platform, page)

    def getArticle(self, platform):
        if platform == StockPlatForm.XQ:
            mainReStr = '//div[@class="container article__container"]//article[@class="article__bd"]//div[@class="article__bd__detail"]'

            self.articleContent(
                "雪球网",
                mainRe=mainReStr,
                reChild=False,
            )
        elif platform == StockPlatForm.TG:
            mainReStr = '//div[@class="content-left left p_wenz"]//div[@class="article-content"]//div[@class="article-text p_coten"]'

            self.articleContent(
                "淘股吧",
                mainRe=mainReStr,
                reChild=False,
            )

    def articleContent(self, platform, mainRe, reChild=True):
        """
        文章正文
        """
        newArticle = self.sqlite.query_many(
            "SELECT id,url,time FROM article_stock WHERE article = '' and platform = '"
            + platform + "' and time > " + str(self.startTime))

        if newArticle is not None and len(newArticle) != 0:
            for item in newArticle:
                try:
                    content = self.request.get(item[1], isBytes=False)
                    article = self.request.getXpath(mainRe, content)

                    if reChild:
                        results = self.request.getXpath(
                            '//p/strong/text()', article[0])
                        if len(results) == 0:
                            results = self.request.getXpath(
                                '//p/text()', article[0])
                        value = re.sub(r"\s+", "", "".join(results))
                    else:
                        value = re.sub(
                            '<.*?>',
                            "",
                            HTMLParser().unescape(article[0].decode()),
                        )

                    sameContent = []
                    contentList = self.sqlite.query_many(
                        "SELECT article FROM article_stock WHERE article != ''"
                    )
                    if contentList is not None and len(contentList) != 0:
                        sameContent = [
                            item
                            for item in contentList if difflib.SequenceMatcher(
                                None, item[0], value).quick_ratio() > 0.9
                        ]
                    if len(sameContent) == 0:
                        self.sqlite.operate_one(
                            "update article_stock set article=? where id=?",
                            (value, item[0]))
                except Exception as e:
                    saveLog("序号" + str(item[0]) + "文章内容解析错误")
                # time.sleep(3)

    def codeFind(self, platform):
        """
        股票查询
        """
        self.sqlite.delete_record(
            "DELETE FROM article_stock WHERE article='' and platform='" +
            platform + "'")

        stockList = self.sqlite.query_many("SELECT name,code FROM code_stock")
        articleValue = self.sqlite.query_many(
            "SELECT article FROM article_stock WHERE article != '' and platform = '"
            + platform + "'")

        stocks = []
        for item in articleValue:
            stocks.extend([
                stock for stock in stockList
                if stock[0] in item[0] or stock[1] in item[0]
            ])

        stockCounts = {}
        for item in stocks:
            stockName, stockCode = item
            if stockName in stockCounts:
                stockCounts[stockName] += 1
            else:
                stockCounts[stockName] = 1
        return sorted(stockCounts.items(), key=lambda x: x[1], reverse=True)

    def timeChange(self, time):
        timeNow = datetime.datetime.now()

        if "今天" in time:
            month = "0" + str(timeNow.month) if timeNow.month < 10 else str(
                timeNow.month)

            day = "0" + str(timeNow.day) if timeNow.day < 10 else str(
                timeNow.day)

            return time.replace("今天", month + "-" + day)
        elif "分钟前" in time:
            month = "0" + str(timeNow.month) if timeNow.month < 10 else str(
                timeNow.month)

            day = "0" + str(timeNow.day) if timeNow.day < 10 else str(
                timeNow.day)

            timeStr = month + "-" + day + " " + str(timeNow.hour) + ":00"
            return timeStr
        else:
            return time
